{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import html\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>location</th>\n",
       "      <th>department</th>\n",
       "      <th>salary_range</th>\n",
       "      <th>company_profile</th>\n",
       "      <th>description</th>\n",
       "      <th>requirements</th>\n",
       "      <th>benefits</th>\n",
       "      <th>telecommuting</th>\n",
       "      <th>has_company_logo</th>\n",
       "      <th>has_questions</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>required_experience</th>\n",
       "      <th>required_education</th>\n",
       "      <th>industry</th>\n",
       "      <th>function</th>\n",
       "      <th>fraudulent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Office Manager</td>\n",
       "      <td>GR, I, Athens</td>\n",
       "      <td>Administration</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Workable is a venture-backed startup making cl...</td>\n",
       "      <td>We are looking for an experienced office manag...</td>\n",
       "      <td>Excellent use of English, speaking and writing...</td>\n",
       "      <td>Our goal is to create a company where employee...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Front End Engineer</td>\n",
       "      <td>US, CA, Emeryville</td>\n",
       "      <td>Engineering</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Abakus is a software platform built on game th...</td>\n",
       "      <td>As Front End Engineer you will be leading the ...</td>\n",
       "      <td>3-5 years of UI/UX development experience with...</td>\n",
       "      <td>Competitive salaryStock optionsComprehensive b...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Software</td>\n",
       "      <td>Engineering</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Software Engineer</td>\n",
       "      <td>US, MA, Wilmington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0-130000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our client is one of the leading SaaS-based pr...</td>\n",
       "      <td>At least 5 years experience developing large-s...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Product Manager</td>\n",
       "      <td>US, CA, Manhattan Beach</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Boston Consulting Group (#URL_45423e1e1670...</td>\n",
       "      <td>We are looking for a Product Manager to be a p...</td>\n",
       "      <td>BASIC JOB REQUIREMENTS:Bachelors Degree form a...</td>\n",
       "      <td>The Boston Consulting Group (BCG) is a global ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Associate</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>Computer Software</td>\n",
       "      <td>Product Management</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sales Intern</td>\n",
       "      <td>US, NY, New York</td>\n",
       "      <td>Sales</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Lean Startup Machine has trained over 25,000 a...</td>\n",
       "      <td>Experience in sales preferredInterest in tech ...</td>\n",
       "      <td>Be part of a growing and global team that has ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Part-time</td>\n",
       "      <td>Internship</td>\n",
       "      <td>Some College Coursework Completed</td>\n",
       "      <td>Management Consulting</td>\n",
       "      <td>Sales</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                title                 location      department salary_range  \\\n",
       "0      Office Manager            GR, I, Athens  Administration          NaN   \n",
       "1  Front End Engineer       US, CA, Emeryville     Engineering          NaN   \n",
       "2   Software Engineer       US, MA, Wilmington             NaN     0-130000   \n",
       "3     Product Manager  US, CA, Manhattan Beach             NaN          NaN   \n",
       "4        Sales Intern         US, NY, New York           Sales          NaN   \n",
       "\n",
       "                                     company_profile  \\\n",
       "0  Workable is a venture-backed startup making cl...   \n",
       "1  Abakus is a software platform built on game th...   \n",
       "2                                                NaN   \n",
       "3  The Boston Consulting Group (#URL_45423e1e1670...   \n",
       "4                                                NaN   \n",
       "\n",
       "                                         description  \\\n",
       "0  We are looking for an experienced office manag...   \n",
       "1  As Front End Engineer you will be leading the ...   \n",
       "2  Our client is one of the leading SaaS-based pr...   \n",
       "3  We are looking for a Product Manager to be a p...   \n",
       "4  Lean Startup Machine has trained over 25,000 a...   \n",
       "\n",
       "                                        requirements  \\\n",
       "0  Excellent use of English, speaking and writing...   \n",
       "1  3-5 years of UI/UX development experience with...   \n",
       "2  At least 5 years experience developing large-s...   \n",
       "3  BASIC JOB REQUIREMENTS:Bachelors Degree form a...   \n",
       "4  Experience in sales preferredInterest in tech ...   \n",
       "\n",
       "                                            benefits  telecommuting  \\\n",
       "0  Our goal is to create a company where employee...              0   \n",
       "1  Competitive salaryStock optionsComprehensive b...              0   \n",
       "2                                                NaN              0   \n",
       "3  The Boston Consulting Group (BCG) is a global ...              0   \n",
       "4  Be part of a growing and global team that has ...              0   \n",
       "\n",
       "   has_company_logo  has_questions employment_type required_experience  \\\n",
       "0                 1              1             NaN                 NaN   \n",
       "1                 1              1       Full-time    Mid-Senior level   \n",
       "2                 0              0       Full-time                 NaN   \n",
       "3                 1              0       Full-time           Associate   \n",
       "4                 0              1       Part-time          Internship   \n",
       "\n",
       "                  required_education               industry  \\\n",
       "0                                NaN                    NaN   \n",
       "1                                NaN      Computer Software   \n",
       "2                  Bachelor's Degree                    NaN   \n",
       "3                  Bachelor's Degree      Computer Software   \n",
       "4  Some College Coursework Completed  Management Consulting   \n",
       "\n",
       "             function  fraudulent  \n",
       "0                 NaN           0  \n",
       "1         Engineering           0  \n",
       "2                 NaN           0  \n",
       "3  Product Management           0  \n",
       "4               Sales           0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv('../data/train.csv')\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>location</th>\n",
       "      <th>department</th>\n",
       "      <th>salary_range</th>\n",
       "      <th>company_profile</th>\n",
       "      <th>description</th>\n",
       "      <th>requirements</th>\n",
       "      <th>benefits</th>\n",
       "      <th>telecommuting</th>\n",
       "      <th>has_company_logo</th>\n",
       "      <th>has_questions</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>required_experience</th>\n",
       "      <th>required_education</th>\n",
       "      <th>industry</th>\n",
       "      <th>function</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Optician / Sales</td>\n",
       "      <td>US, MD, Bethesda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>We are a long term well established Optometric...</td>\n",
       "      <td>The ideal person should be highly self motivat...</td>\n",
       "      <td>We offer a competitive compensation package, w...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Health, Wellness and Fitness</td>\n",
       "      <td>Sales</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Payroll Data Coordinator Positions - Earn $100...</td>\n",
       "      <td>US, TX, Abbott</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>We are a full-service marketing and staffing f...</td>\n",
       "      <td>RequirementsAll you need is access to the Inte...</td>\n",
       "      <td>This is an entry level position and we offer f...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dispatcher</td>\n",
       "      <td>US, TX, AUSTIN</td>\n",
       "      <td>CALLCENTER TECH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our globally connected world has forced busine...</td>\n",
       "      <td>We currently have an opening for a Dispatcher ...</td>\n",
       "      <td>·High School diploma/GEDSuccessfully complete ...</td>\n",
       "      <td>POSITION TYPE: FULL TIMEHEALTH CARE, DENTAL , ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>High School or equivalent</td>\n",
       "      <td>Retail</td>\n",
       "      <td>Customer Service</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Content Marketing Manager</td>\n",
       "      <td>US, NY, New York</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Indicative is changing the way internet busine...</td>\n",
       "      <td>Indicative is changing the way internet busine...</td>\n",
       "      <td>Essential:Unparalleled ability to communicate ...</td>\n",
       "      <td>Great healthcare and employee benefits, top of...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Associate</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Marketing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Talent Sourcing Specialist</td>\n",
       "      <td>NZ, N, Auckland</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EROAD was established to modernise New Zealand...</td>\n",
       "      <td>EROAD is a fully integrated technology, road c...</td>\n",
       "      <td>Ideally you will have several years’ experienc...</td>\n",
       "      <td>Competitive salary and package.Flexibility - h...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Associate</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Information Technology and Services</td>\n",
       "      <td>Human Resources</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title          location  \\\n",
       "0                                   Optician / Sales  US, MD, Bethesda   \n",
       "1  Payroll Data Coordinator Positions - Earn $100...   US, TX, Abbott    \n",
       "2                                         Dispatcher    US, TX, AUSTIN   \n",
       "3                          Content Marketing Manager  US, NY, New York   \n",
       "4                         Talent Sourcing Specialist   NZ, N, Auckland   \n",
       "\n",
       "        department salary_range  \\\n",
       "0              NaN          NaN   \n",
       "1              NaN          NaN   \n",
       "2  CALLCENTER TECH          NaN   \n",
       "3        Marketing          NaN   \n",
       "4              NaN          NaN   \n",
       "\n",
       "                                     company_profile  \\\n",
       "0                                                NaN   \n",
       "1                                                NaN   \n",
       "2  Our globally connected world has forced busine...   \n",
       "3  Indicative is changing the way internet busine...   \n",
       "4  EROAD was established to modernise New Zealand...   \n",
       "\n",
       "                                         description  \\\n",
       "0  We are a long term well established Optometric...   \n",
       "1  We are a full-service marketing and staffing f...   \n",
       "2  We currently have an opening for a Dispatcher ...   \n",
       "3  Indicative is changing the way internet busine...   \n",
       "4  EROAD is a fully integrated technology, road c...   \n",
       "\n",
       "                                        requirements  \\\n",
       "0  The ideal person should be highly self motivat...   \n",
       "1  RequirementsAll you need is access to the Inte...   \n",
       "2  ·High School diploma/GEDSuccessfully complete ...   \n",
       "3  Essential:Unparalleled ability to communicate ...   \n",
       "4  Ideally you will have several years’ experienc...   \n",
       "\n",
       "                                            benefits  telecommuting  \\\n",
       "0  We offer a competitive compensation package, w...              0   \n",
       "1  This is an entry level position and we offer f...              1   \n",
       "2  POSITION TYPE: FULL TIMEHEALTH CARE, DENTAL , ...              0   \n",
       "3  Great healthcare and employee benefits, top of...              0   \n",
       "4  Competitive salary and package.Flexibility - h...              0   \n",
       "\n",
       "   has_company_logo  has_questions employment_type required_experience  \\\n",
       "0                 0              0       Full-time                 NaN   \n",
       "1                 0              0             NaN                 NaN   \n",
       "2                 1              1       Full-time         Entry level   \n",
       "3                 1              1       Full-time           Associate   \n",
       "4                 1              0       Full-time           Associate   \n",
       "\n",
       "          required_education                             industry  \\\n",
       "0                        NaN         Health, Wellness and Fitness   \n",
       "1                        NaN                                  NaN   \n",
       "2  High School or equivalent                               Retail   \n",
       "3          Bachelor's Degree                                  NaN   \n",
       "4                Unspecified  Information Technology and Services   \n",
       "\n",
       "           function  \n",
       "0             Sales  \n",
       "1               NaN  \n",
       "2  Customer Service  \n",
       "3         Marketing  \n",
       "4   Human Resources  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv('../data/test.csv')\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = train_df.columns.tolist()\n",
    "columns.remove('fraudulent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title',\n",
       " 'location',\n",
       " 'department',\n",
       " 'salary_range',\n",
       " 'company_profile',\n",
       " 'description',\n",
       " 'requirements',\n",
       " 'benefits',\n",
       " 'telecommuting',\n",
       " 'has_company_logo',\n",
       " 'has_questions',\n",
       " 'employment_type',\n",
       " 'required_experience',\n",
       " 'required_education',\n",
       " 'industry',\n",
       " 'function']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve(row):\n",
    "    text = ''\n",
    "    for column in columns:\n",
    "        if column in ['company_profile', 'description', 'requirements', 'benefits']:\n",
    "            text += (column + '' + str(row[column])[:100] + ' ')\n",
    "        else:\n",
    "            text += (column + '' + str(row[column]) + ' ')\n",
    "    return text[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['text'] = train_df.apply(lambda x: solve(x), axis=1)\n",
    "test_df['text'] = test_df.apply(lambda x: solve(x), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "84"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# texts = train_df[pd.notnull(train_df['company_profile'])].company_profile.tolist()\n",
    "# max:884, 0.9:209\n",
    "# texts = train_df[pd.notnull(train_df['description'])].description.tolist()\n",
    "# max:2074, 0.9:324\n",
    "# texts = train_df[pd.notnull(train_df['requirements'])].requirements.tolist()\n",
    "# max:1612, 0.9:180\n",
    "# texts = train_df[pd.notnull(train_df['benefits'])].benefits.tolist()\n",
    "# max:678, 0.9:116\n",
    "# 截取前，max:2621, 0.9:659\n",
    "# 截取后，max:236, 0.9:84\n",
    "texts = train_df.text.tolist()\n",
    "max_len = 0\n",
    "df_len_list = []\n",
    "for i in range(len(texts)):\n",
    "    text = []\n",
    "    text = texts[i].split(' ')\n",
    "    df_len_list.append(len(text))\n",
    "    max_len = max(max_len, len(text))\n",
    "print(max_len)\n",
    "sorted_len = sorted(df_len_list)\n",
    "sorted_len[int(len(sorted_len)*0.9)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    16914\n",
       "1      766\n",
       "Name: fraudulent, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.fraudulent.value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
